{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyzing KITTI-tracking data and putting it in a table\n",
    "\n",
    "(And maybe boxplots if sensible.)\n",
    "\n",
    "Warning: copypasta from the 'StaticDepthAnalysis' notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import os\n",
    "import sys\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "from matplotlib import rc\n",
    "# Enable full LaTeX support in plot text. Requires a full-fledged LaTeX installation\n",
    "# on your system, accessible via PATH.\n",
    "rc('text', usetex=True)\n",
    "\n",
    "plt.rcParams[\"figure.figsize\"] = (16, 6)\n",
    "matplotlib.rcParams.update({'font.size': 16})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input-dispnet No-dyn & Fusion-dispnet No-dyn &Fusion-dispnet Dyn &Input-elas No-dyn & Fusion-elas No-dyn &Fusion-elas Dyn &\n",
      "00 &        0.8546   &         0.8402   & \\textbf{0.8743}&\n",
      "            0.8512   &         0.8285   & \\textbf{0.8650}\\\\\n",
      "01 &        0.9160   & \\textbf{0.9266}&         0.9215   &\n",
      "            0.9218   & \\textbf{0.9373}&         0.9316   \\\\\n",
      "02 &        0.9563   &         0.9635   & \\textbf{0.9635}&\n",
      "            0.9677   &         0.9790   & \\textbf{0.9797}\\\\\n",
      "03 &        0.8455   &         0.8522   & \\textbf{0.8591}&\n",
      "            0.8875   &         0.8987   & \\textbf{0.9062}\\\\\n",
      "04 &        0.8965   & \\textbf{0.9069}&         0.9067   &\n",
      "            0.9117   &         0.9357   & \\textbf{0.9385}\\\\\n",
      "05 &        0.8393   &         0.8354   & \\textbf{0.8400}&\n",
      "            0.8990   &         0.9102   & \\textbf{0.9163}\\\\\n",
      "06 &        0.9523   &         0.9667   & \\textbf{0.9707}&\n",
      "            0.9602   &         0.9668   & \\textbf{0.9709}\\\\\n",
      "07 &        0.8743   &         0.8825   & \\textbf{0.8838}&\n",
      "            0.9009   &         0.9222   & \\textbf{0.9251}\\\\\n",
      "08 &        0.9029   &         0.9086   & \\textbf{0.9123}&\n",
      "            0.9267   &         0.9282   & \\textbf{0.9311}\\\\\n",
      "09 &\\textbf{0.8824}&         0.8755   &         0.8786   &\n",
      "            0.9184   &         0.9210   & \\textbf{0.9238}\\\\\n",
      "\n",
      "\n",
      "\n",
      "input-dispnet-NO-dynamic 802 802\n",
      "Mean accuracy: 0.8824137465784287, Mean completeness: 0.9850301072321747\n",
      "input-dispnet-dynamic-mode 802 802\n",
      "Mean accuracy: 0.8848250963161465, Mean completeness: 0.9850301072321747\n",
      "input-elas-NO-dynamic 802 802\n",
      "Mean accuracy: 0.9183723940605802, Mean completeness: 0.8351923739860994\n",
      "input-elas-dynamic-mode 802 802\n",
      "Mean accuracy: 0.9206006985875052, Mean completeness: 0.8351923739860994\n",
      "fusion-dispnet-NO-dynamic 802 802\n",
      "Mean accuracy: 0.8754851357927843, Mean completeness: 0.9840660981747308\n",
      "fusion-dispnet-dynamic-mode 802 802\n",
      "Mean accuracy: 0.8785697171986308, Mean completeness: 0.9734293638938042\n",
      "fusion-elas-NO-dynamic 802 802\n",
      "Mean accuracy: 0.9210006147867821, Mean completeness: 0.9735208636406815\n",
      "fusion-elas-dynamic-mode 802 802\n",
      "Mean accuracy: 0.923813080263282, Mean completeness: 0.9610367366722521\n"
     ]
    }
   ],
   "source": [
    "out_dir = '../fig'\n",
    "\n",
    "def gen_plots(root, part, eval_completeness):\n",
    "    # if 'eval_completeness' is false, eval accuracy.\n",
    "    # TODO maybe also compute results for dynamic parts.\n",
    "    file_pattern = 'k-99999-kitti-tracking-sequence-{sequence_id:04d}--offset-0-'    \\\n",
    "                   'depth-precomputed-{depth}-voxelsize-0.0500-max-depth-m-20.00-'  \\\n",
    "                   '{fusion}-NO-direct-ref-with-fusion-weights-{part}.csv'\n",
    "    base = os.path.join(root, file_pattern)\n",
    "    res = {}\n",
    "    res_completeness = {}\n",
    "    \n",
    "    sequences = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]\n",
    "    fusions = ['NO-dynamic', 'dynamic-mode']\n",
    "    metrics = ['input', 'fusion']\n",
    "    depths = ['dispnet', 'elas']\n",
    "    \n",
    "    res = {}\n",
    "    res_completeness = {}\n",
    "    \n",
    "    # Not a proper header. The real one is in the thesis tex, since it's nested\n",
    "    # and pointless to generate from code.\n",
    "    for depth in depths:\n",
    "        for fusion in fusions:\n",
    "            for metric in metrics:\n",
    "                dyn_str = 'Dyn' if fusion == 'dynamic-mode' else 'No-dyn'\n",
    "                if metric == 'input' and fusion == 'NO-dynamic':\n",
    "                    print('Input-{} {} & '.format(depth, dyn_str), end='')\n",
    "                    \n",
    "                if metric == 'fusion':\n",
    "                    print('Fusion-{} {} &'.format(depth, dyn_str), end='')\n",
    "    print()\n",
    "    \n",
    "    acc_perc_agg = {}\n",
    "    completeness_agg = {}\n",
    "    \n",
    "    # Yowza, that's a lot of loop nesting!\n",
    "    for sequence_id in sequences:\n",
    "        seq_count = -1\n",
    "        \n",
    "        print('{:02d} &'.format(sequence_id), end='')\n",
    "        for depth in depths:\n",
    "            best_key = None\n",
    "            best_score = -1.0\n",
    "            \n",
    "            for fusion in fusions:\n",
    "                fname = base.format(sequence_id=sequence_id, depth=depth, \n",
    "                                        fusion=fusion, part=part)\n",
    "                df = pd.read_csv(fname)\n",
    "            \n",
    "                for metric in metrics:\n",
    "                    key = \"{}-{}-{}-{:02d}\".format(metric, depth, fusion, sequence_id)\n",
    "                    cross_seq_key = \"{}-{}-{}\".format(metric, depth, fusion)\n",
    "\n",
    "                    # Do not count frames with no pixels in them. This would distort the \n",
    "                    # dynamic reconstruction metrics due to frames containing no objects.\n",
    "                    ok = (df['{}-total-3.00-kitti'.format(metric)] != 0)\n",
    "\n",
    "                    err = df['{}-error-3.00-kitti'.format(metric)][ok]\n",
    "                    tot = df['{}-total-3.00-kitti'.format(metric)][ok]\n",
    "                    mis = df['{}-missing-3.00-kitti'.format(metric)][ok]\n",
    "                    cor = df['{}-correct-3.00-kitti'.format(metric)][ok]\n",
    "                    mis_sep = df['{}-missing-separate-3.00-kitti'.format(metric)][ok]\n",
    "\n",
    "                    acc_perc = cor / (tot - mis)\n",
    "                    # When evaluating dynamic parts, sometimes we encounter cases with\n",
    "                    # e.g., very distant cars where tot == mis.\n",
    "                    acc_perc = acc_perc[~np.isnan(acc_perc)]\n",
    "                    completeness = 1.0 - (mis_sep / tot)\n",
    "                    \n",
    "                    if cross_seq_key not in acc_perc:\n",
    "                        acc_perc_agg[cross_seq_key] = []\n",
    "                        completeness_agg[cross_seq_key] = []\n",
    "                        \n",
    "                    acc_perc_agg[cross_seq_key] += acc_perc.tolist()\n",
    "                    completeness_agg[cross_seq_key] += completeness.tolist()\n",
    "                    \n",
    "                    if eval_completeness:\n",
    "                        res[key] = completeness\n",
    "                    else:\n",
    "                        res[key] = acc_perc\n",
    "                    \n",
    "                    \n",
    "                    mean_acc_perc = acc_perc.mean()\n",
    "                    mean_com_perc = completeness.mean()\n",
    "                    \n",
    "                    # The input should be the same in dynamic and non-dynamic mode.\n",
    "                    if not (metric == 'input' and fusion == 'dynamic-mode'):\n",
    "                        if eval_completeness:\n",
    "                            # Compute and display completeness\n",
    "                            if mean_com_perc > best_score:\n",
    "                                best_score = mean_com_perc\n",
    "                                best_key = key\n",
    "                        else:\n",
    "                            # Compute and display accuracy\n",
    "                            if mean_acc_perc > best_score:\n",
    "                                best_score = mean_acc_perc\n",
    "                                best_key = key\n",
    "                    \n",
    "                    if -1 == seq_count:\n",
    "                        seq_count = len(df)\n",
    "                    elif seq_count != len(df):\n",
    "                        print(\"Warning: inconsistent lengths for sequence {:04d}\".format(sequence_id))\n",
    "                        print(sequence_id, depth, fusion, metric, len(df))\n",
    "                  \n",
    "            for fusion in fusions:\n",
    "                for metric in metrics:\n",
    "                    key = \"{}-{}-{}-{:02d}\".format(metric, depth, fusion, sequence_id)\n",
    "\n",
    "                    if not (metric == 'input' and fusion == 'dynamic-mode'):\n",
    "                        if res[key].mean() is np.nan:\n",
    "                            # No data for the dynamic parts when doing standard fusion!\n",
    "                            assert(fusion == 'NO-dynamic')\n",
    "                            continue\n",
    "                        elif key == best_key:\n",
    "                            print(r'\\textbf{{{:.4f}}}'.format(res[key].mean()), end='')\n",
    "                        else:\n",
    "                            print(r'        {:.4f}   '.format(res[key].mean()), end='')\n",
    "                            \n",
    "                        if not (metric == 'fusion' and fusion == 'dynamic-mode'):\n",
    "                            print('& ', end='')\n",
    "                     \n",
    "            if depth == depths[0]:\n",
    "                print('&', end='\\n    ')\n",
    "            \n",
    "        print(r'\\\\')\n",
    "        \n",
    "    print(\"\\n\\n\")\n",
    "    for metric in metrics:\n",
    "        for depth in depths:\n",
    "            for fusion in fusions:\n",
    "                key = \"{}-{}-{}\".format(metric, depth, fusion)\n",
    "                acc_perc = acc_perc_agg[key]\n",
    "                completeness = completeness_agg[key]\n",
    "                print(key, len(acc_perc), len(completeness))\n",
    "                print(\"Mean accuracy: {}, Mean completeness: {}\".format(np.mean(acc_perc), np.mean(completeness)))\n",
    "\n",
    "#                 box_colors.append(colors[depth][metric])\n",
    "#                 columns.append(key)\n",
    "#                 box_positions.append(box_offset)\n",
    "    \n",
    "gen_plots('../csv/tracking-res/', 'static-depth-result', eval_completeness=False)\n",
    "# gen_plots('../csv/tracking-res/', 'dynamic-depth-result')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gen_baseline_data(root, part, eval_completeness):\n",
    "    print(\"TODO(andrei): Same as above but for the InfiniTAM baseline.\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
